﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using HtmlAgilityPack;
using NewsMine.DomainObjects;
using System.Web;
using System.Net.Cache;
using System.IO.Compression;
using System.Xml;
using System.ServiceModel.Syndication;
using System.Runtime.Serialization.Formatters.Binary;

namespace NewsMine.Utilities
{
    public class WebHelper
    {
        public static string DownloadHtmlFromUrl(string url)
        {
            return DownLoadHttpTextByCompressionEnabled(url); //putting for test...

            try
            {
                TempLogger.Log("Starting method => DownloadHtmlFromUrl method, Url is :" + url);

                WebRequest request = WebRequest.Create(url);

                //todo: needs to be tweaked and read from the configuration.
                request.Timeout = 30 * 1000;

                WebResponse response = request.GetResponse();
                Stream data = response.GetResponseStream();
                string html = String.Empty;
                using (StreamReader sr = new StreamReader(data))
                {
                    html = sr.ReadToEnd();
                }


                return html;

            }
            catch (WebException wEx)
            {
                TempLogger.LogWebException(url, wEx);
                throw wEx;
            }
            catch (Exception ex)
            {
                TempLogger.Log("Error in DownloadHtmlFromUrl method, Url is :" + url, ex);
                throw ex;
            }

        }

        //todo: this needs to be used;
        public static string DownLoadHttpTextByCompressionEnabled(string URL)
        {
              try
            {
                TempLogger.Log("Starting method => DownLoadHttpTextByCompressionEnabled method, Url is :" + URL);

            string JSON;
            // Create the web request  
            HttpWebRequest request = WebRequest.Create(URL) as HttpWebRequest;
            HttpRequestCachePolicy cPolicy = new HttpRequestCachePolicy(HttpRequestCacheLevel.Revalidate);

            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            //request.Accept = "application/json";
            request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
            request.CachePolicy = cPolicy;
            request.Pipelined = false;

            request.Timeout = 30 * 1000;
            // Get response  
            using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
            {
                Stream responseStream = responseStream = response.GetResponseStream();

                if (response.ContentEncoding.ToLower().Contains("gzip"))
                    responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
                else if (response.ContentEncoding.ToLower().Contains("deflate"))
                    responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);


                // Get the response stream  
                StreamReader readerF = new StreamReader(responseStream);
                JSON = readerF.ReadToEnd();
            }

            return JSON;
            }
              catch (WebException wEx)
              {
                  TempLogger.LogWebException(URL, wEx);
                  throw wEx;
              }
              catch (Exception ex)
              {
                  TempLogger.Log("Error in DownLoadHttpTextByCompressionEnabled method, Url is :" + URL, ex);
                  throw ex;
              }
        }

        public static HtmlContentFromUrl DownLoadHttpTextByCompressionEnabledWithExtraInfo(string URL)
        {
            try
            {
                HtmlContentFromUrl resultFromUrl = new HtmlContentFromUrl();

                resultFromUrl.Url = URL;

                TempLogger.Log("Starting method => DownLoadHttpTextByCompressionEnabledWithExtraInfo method, Url is :" + URL);

                string responseText;
                // Create the web request  
                HttpWebRequest request = WebRequest.Create(URL) as HttpWebRequest;
                HttpRequestCachePolicy cPolicy = new HttpRequestCachePolicy(HttpRequestCacheLevel.Revalidate);

                request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
                request.UserAgent = "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)";
                //request.Accept = "application/json";
                request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                request.CachePolicy = cPolicy;
                request.Pipelined = false;

                request.Timeout = 30 * 1000;
                // Get response  
                using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
                {
                    Stream responseStream = responseStream = response.GetResponseStream();

                    resultFromUrl.ContentType = response.ContentType;
                    resultFromUrl.FinalUrl = response.ResponseUri.ToString();


                    if (response.ContentEncoding.ToLower().Contains("gzip"))
                        responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))
                        responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);


                    // Get the response stream  
                    StreamReader readerF = new StreamReader(responseStream);
                    responseText = readerF.ReadToEnd();

                    resultFromUrl.HtmlContent = responseText;
                }

                return resultFromUrl;
            }
            catch (WebException wEx)
            {
                TempLogger.LogWebException(URL, wEx);
                throw wEx;
            }
            catch (Exception ex)
            {
                TempLogger.Log("Error in DownLoadHttpTextByCompressionEnabled method, Url is :" + URL, ex);
                throw ex;
            }
        }

        public static object DownLoadHttpDataByCompressionEnabledWithExtraInfo(string URL)
        {
            try
            {
                byte[] resultBytes;

               

                TempLogger.Log("Starting method => DownLoadHttpTextByCompressionEnabledWithExtraInfo method, Url is :" + URL);

                string responseText;
                // Create the web request  
                HttpWebRequest request = WebRequest.Create(URL) as HttpWebRequest;
                HttpRequestCachePolicy cPolicy = new HttpRequestCachePolicy(HttpRequestCacheLevel.Revalidate);

                request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
                request.UserAgent = "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)";
                //request.Accept = "application/json";
                request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                request.CachePolicy = cPolicy;
                request.Pipelined = false;

                request.Timeout = 30 * 1000;
                // Get response  
                using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
                {
                    Stream responseStream = responseStream = response.GetResponseStream();

                    if (response.ContentEncoding.ToLower().Contains("gzip"))
                        responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))
                        responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);



                    BinaryReader bReader = new BinaryReader(responseStream);

                    var bytes = bReader.ReadBytes(Convert.ToInt32(responseStream.Length));

                    //BinaryFormatter bFormatter = new BinaryFormatter();

                    object obj = Utilities.ObjectByteArrayConverter.ByteArrayToObject(bytes);

                    return obj;
                    // Get the response stream  
                    //StreamReader readerF = new StreamReader(responseStream);
                    //responseText = readerF.ReadToEnd();

                    //resultFromUrl.HtmlContent = responseText;
                }

            }
            catch (WebException wEx)
            {
                TempLogger.LogWebException(URL, wEx);
                throw wEx;
            }
            catch (Exception ex)
            {
                TempLogger.Log("Error in DownLoadHttpTextByCompressionEnabled method, Url is :" + URL, ex);
                throw ex;
            }
        }

        public static List<HtmlLink> GetHtmlLinks(string baseUrl, HtmlContentFromUrl htmlContentFromUrl)
        {
            //todo: needs to have if condition, by checking content type whether it is html or rss feed, just call appropriate methods.

            if (!string.IsNullOrEmpty(htmlContentFromUrl.ContentType) &&
                (
                htmlContentFromUrl.ContentType.ToLower().Contains("rss") ||
                htmlContentFromUrl.ContentType.ToLower().Contains("xml") ||
                htmlContentFromUrl.ContentType.ToLower().Contains("atom"))
                )
            {
                return GetHtmlLinksFromFeed(baseUrl, htmlContentFromUrl);
            }


            return GetHtmlLinks(baseUrl, htmlContentFromUrl.HtmlContent);
        }

        private static List<HtmlLink> GetHtmlLinksFromFeed(string baseUrl, HtmlContentFromUrl htmlContentFromUrl)
        {
            // convert string to stream
            byte[] byteArray = Encoding.UTF8.GetBytes(htmlContentFromUrl.HtmlContent);
            //byte[] byteArray = Encoding.ASCII.GetBytes(contents);
            MemoryStream stream = new MemoryStream(byteArray);

            var reader = XmlReader.Create(stream);

            SyndicationFeed feed = SyndicationFeed.Load(reader);


            reader.Close();
            stream.Close();
            stream.Dispose();

            List<HtmlLink> results = new List<HtmlLink>();

            foreach (SyndicationItem item in feed.Items)
            {
                HtmlLink htmlLink = new HtmlLink();
                htmlLink.IsFeedSourced = true;

                if (item.Title != null && !string.IsNullOrEmpty(item.Title.Text))
                {
                    htmlLink.Title = item.Title.Text;
                }
                else
                {
                    //if there is no title, no point adding this item. so let skip this.
                    continue;
                }

                if (item.Links != null && item.Links.Count > 0)
                {
                    htmlLink.Link = item.Links[0].Uri.ToString();
                }

                if (item.Summary != null && !string.IsNullOrEmpty(item.Summary.Text))
                {
                    htmlLink.Description = item.Summary.Text;
                }

                //here we are trying luck to check for description from the content property, if summary is not given us.
                if (string.IsNullOrEmpty(htmlLink.Description))
                {
                    if (item.Content != null && !string.IsNullOrEmpty(item.Content.ToString()))
                    {
                        htmlLink.Description = item.Content.ToString();
                    }
                }

                if (item.Categories != null && item.Categories.Count > 0)
                {
                    foreach (var cat in item.Categories)
                    {
                        if (!string.IsNullOrEmpty(cat.Name))
                            htmlLink.CategoriesFromFeed.Add(cat.Name);
                    }

                    //once all things are done, just remove the , from end of the string.
                }

                htmlLink.PubDate = Convert.ToDateTime(item.PublishDate.ToString());

                //htmlLink.PubDate = DateTime.Parse(item.PublishDate.ToString(), null, System.Globalization.DateTimeStyles.AssumeUniversal);

                //todo: needs to capture the description ( content / summary) into some field;
                //I am not sure how to extract from the syndicationItem. here the guess is content or summary field contains description.

                results.Add(htmlLink);
            }

            return results;
        }

        public static List<HtmlLink> GetHtmlLinks(string baseUrl, string htmlContent)
        {
            HtmlDocument hdDoc = new HtmlDocument();
            hdDoc.LoadHtml(htmlContent);

            return GetHtmlLinks(baseUrl, hdDoc);
        }


        public static List<HtmlLink> GetHtmlLinks(string baseUrl, HtmlDocument hdDoc, string xpath)
        {

            ConvertLinksFromRelatetiveToAbsolute(baseUrl, hdDoc);

            List<HtmlLink> lsLinks = new List<HtmlLink>();

            var linkNodes = hdDoc.DocumentNode.SelectNodes(xpath);

            if (linkNodes == null)
                return new List<HtmlLink>();

            foreach (HtmlNode link in linkNodes)
            {

                string hrefValue = link.Attributes["href"].Value;

                string hrefOriginaValue = hrefValue;

                hrefValue = hrefValue.ToLower();

                if (!string.IsNullOrEmpty(hrefValue) && !hrefValue.StartsWith("javascript") && (hrefValue.StartsWith("http://") || !hrefValue.StartsWith("https://")))
                {
                    HtmlLink htmlLink = new HtmlLink { Link = hrefOriginaValue };
                    //htmlLink.Title = HttpUtility.HtmlDecode(link.InnerText);
                    htmlLink.Title = link.InnerText;
                    if (!string.IsNullOrEmpty(htmlLink.Title))
                    {
                        htmlLink.Title = HttpUtility.HtmlDecode(htmlLink.Title).Trim();
                    }
                    //Here we need to get image node, if any existed.

                    if (link.HasChildNodes)
                    {
                        HtmlNode imageNode = null;

                        foreach (var node in link.DescendantNodes())
                        {
                            if (node.Name.ToLower() == "img")
                            {
                                imageNode = node;
                                continue;
                            }
                        }

                        if (imageNode != null)
                        {
                            if (imageNode.Attributes["src"] != null)
                                htmlLink.ImageUrl = imageNode.Attributes["src"].Value;

                            if (imageNode.Attributes["alt"] != null)
                                htmlLink.ImageAltText = imageNode.Attributes["alt"].Value;

                            //if (string.IsNullOrEmpty(htmlLink.Title)
                            //    && !string.IsNullOrEmpty(htmlLink.ImageAltText))
                            //{
                            //    htmlLink.Title = htmlLink.ImageAltText;
                            //}
                        }
                    }


                    // and also if the title is small or empty, we need to look for image's alt text and assigh alt text to title 

                    htmlLink.OuterHtml = link.OuterHtml;
                    htmlLink.HtmlXpath = link.XPath;
                    lsLinks.Add(htmlLink);
                }
                //lsLinks.Add(link.Attributes["href"].Value.ToString());
            }

            return lsLinks;
        }

        public static List<HtmlLink> GetHtmlLinks(string baseUrl, HtmlDocument hdDoc)
        {
            return GetHtmlLinks(baseUrl, hdDoc, "//a[@href]");
        }

        public static List<FeedItem> GetFeedItemsFromHtml(string baseUrl, string htmlContent)
        {
            HtmlDocument hdDoc = new HtmlDocument();
            hdDoc.LoadHtml(htmlContent);

            ConvertLinksFromRelatetiveToAbsolute(baseUrl, hdDoc);

            List<FeedItem> feedItems = new List<FeedItem>();
            foreach (HtmlNode link in hdDoc.DocumentNode.SelectNodes("//a[@href]"))
            {
                string hrefValue = link.Attributes["href"].Value;

                if (!string.IsNullOrEmpty(hrefValue) && !hrefValue.StartsWith("javascript"))
                {
                    FeedItem feedItem = new FeedItem { Link = hrefValue };
                    //feedItem.Title = HttpUtility.HtmlDecode(link.InnerText);
                    feedItem.Title = link.InnerText;
                    if (!string.IsNullOrEmpty(link.InnerText.Trim()))
                    {
                        if (link.SelectNodes("//img").Count == 1)
                        {
                            HtmlNode htmlImageNode = link.SelectNodes("//img")[0];
                            feedItem.ThumbImageUrl = htmlImageNode.Attributes["src"].Value;
                        }
                    }

                    feedItems.Add(feedItem);
                }
                //lsLinks.Add(link.Attributes["href"].Value.ToString());
            }

            Dictionary<string, FeedItem> urlFeeditemDictionaryWithTitle = new Dictionary<string, FeedItem>();
            Dictionary<string, FeedItem> urlFeeditemDictionaryWithImages = new Dictionary<string, FeedItem>();

            foreach (FeedItem feed in feedItems)
            {
                if (string.IsNullOrEmpty(feed.Title) && !string.IsNullOrEmpty(feed.ThumbImageUrl))
                {
                    urlFeeditemDictionaryWithImages.Add(feed.Link, feed);
                }
                else if (!string.IsNullOrEmpty(feed.Title))
                {
                    urlFeeditemDictionaryWithTitle.Add(feed.Link, feed);
                }
            }

            //tobe continue from here..
            // need to map title and images ......
            // finally we need to carry forward the extra images 

            return feedItems;
        }

        public static void ConvertLinksFromRelatetiveToAbsolute(string baseUrl, HtmlDocument hdDoc)
        {
            Uri baseUri = new Uri(baseUrl);

            if (hdDoc == null || hdDoc.DocumentNode==null)
                throw new ApplicationException("HtmlDocument object can't be null to process task, hint=> base url is : " + baseUrl);

            if (hdDoc.DocumentNode == null)
                throw new ApplicationException("HtmlDocument Main node object can't be null to process task , hint=> base url is : " + baseUrl);

            if (hdDoc.DocumentNode.SelectNodes("//*[@href]") == null)
                return;

            foreach (HtmlNode link in hdDoc.DocumentNode.SelectNodes("//*[@href]"))
            {
                var hrefValue = link.Attributes["href"].Value;
                string originalHrefValue = hrefValue;

                if (!string.IsNullOrEmpty(hrefValue))
                {
                    hrefValue = hrefValue.ToLower();
                    if (hrefValue.StartsWith("javascript"))
                        continue;
                    if (hrefValue.StartsWith("mailto"))
                        continue;

                    //if (!hrefValue.StartsWith("../") && !hrefValue.StartsWith("./") && !hrefValue.StartsWith("/") && !hrefValue.StartsWith("http") && !hrefValue.StartsWith("https://"))
                    //    continue;

                    //if ( hrefValue.StartsWith("/") || hrefValue.StartsWith("../") || hrefValue.StartsWith("./"))
                    if(!hrefValue.StartsWith("http") && !hrefValue.StartsWith("https://"))
                    {
                        try
                        {
                            var absoluteUri = new Uri(baseUri, originalHrefValue);
                            link.Attributes["href"].Value = absoluteUri.AbsoluteUri.ToString();
                        }
                        catch
                        {
                           //todo: at least lot it.
                        }
                    }
                }
            }

            if (hdDoc.DocumentNode.SelectNodes("//*[@src]") == null)
                return;

            foreach (HtmlNode link in hdDoc.DocumentNode.SelectNodes("//*[@src]"))
            {
                var hrefValue = link.Attributes["src"].Value;
                var originalHrefValue = hrefValue;

                if (!string.IsNullOrEmpty(hrefValue))
                {
                    hrefValue = hrefValue.ToLower();
                    if (hrefValue.StartsWith("javascript"))
                        continue;

                    //if (hrefValue.StartsWith("/") || hrefValue.StartsWith("../"))

                    if (!hrefValue.StartsWith("http") && !hrefValue.StartsWith("https://"))
                    {
                        try
                        {
                            var absoluteUri = new Uri(baseUri, originalHrefValue);
                            link.Attributes["src"].Value = absoluteUri.AbsoluteUri.ToString();
                        }
                        catch
                        {
                            //todo:need to log this issue.
                        }
                    }
                }
            }
        }

        public static FeedItem GetFeedItemFromTargetUrl(HtmlContentFromUrl htmlContentFromUrl)
        {


            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(htmlContentFromUrl.HtmlContent);

            if (doc.DocumentNode == null)
                return null;

            NewsMine.Utilities.WebHelper.ConvertLinksFromRelatetiveToAbsolute(htmlContentFromUrl.Url, doc);

            FeedItem feedItem = new FeedItem();

            feedItem.Link = htmlContentFromUrl.Url;


            var metaTags = doc.DocumentNode.SelectNodes("//meta");


            if (metaTags != null && metaTags.Count>0)
            {
                foreach (var metaTag in metaTags)
                {
                    if (metaTag.Attributes != null && metaTag.Attributes.Count > 0)
                    {
                        foreach (var attr in metaTag.Attributes)
                        {
                            #region og:xxx tags
                            if (attr.Name.ToLower() == "property")
                            {
                                if (attr.Value == null)
                                    continue;

                                switch (attr.Value.ToLower())
                                {
                                    case null:
                                        continue;
                                        break;
                                    case "og:type":
                                        feedItem.Tags.Add(GetContentFromMetaTag(attr.OwnerNode));
                                        break;
                                    case "og:site_name":

                                        break;
                                    //case "og:title":
                                    //    feedItem.Title = GetContentFromMetaTag(attr.OwnerNode);

                                    //    break;
                                    case "og:description":
                                        feedItem.Description = GetContentFromMetaTag(attr.OwnerNode);
                                        break;

                                    case "og:image":
                                        feedItem.ImageUrl = GetContentFromMetaTag(attr.OwnerNode);
                                        break;

                                    default:
                                        string value = attr.Value.ToLower();

                                        if (value.StartsWith("image") || value.EndsWith("image"))
                                        {
                                            if (string.IsNullOrEmpty(feedItem.ImageUrl))
                                            {
                                                feedItem.ImageUrl = GetContentFromMetaTag(attr.OwnerNode);
                                            }
                                        }
                                        else if ((value.StartsWith("description") || value.EndsWith("description")) && string.IsNullOrEmpty(feedItem.Description))
                                        {
                                            feedItem.Description = GetContentFromMetaTag(attr.OwnerNode);
                                        }

                                        break;
                                }
                            }
                            #endregion

                            #region name tags
                            if (attr.Name.ToLower() == "name")
                            {
                                if (attr.Value == null)
                                    continue;

                                switch (attr.Value.ToLower())
                                {
                                    case null:
                                        continue;
                                        break;
                                    case "author":
                                        feedItem.Author = GetContentFromMetaTag(attr.OwnerNode);
                                        break;
                                    case "description":
                                        if (string.IsNullOrEmpty(feedItem.Description))
                                            feedItem.Description = GetContentFromMetaTag(attr.OwnerNode);
                                        break;
                                    case "keywords":
                                        feedItem.Keyworkds = GetContentFromMetaTag(attr.OwnerNode);
                                        //feedItem.Tags.AddRange(GetContentFromMetaTag(attr.OwnerNode).Split(','));
                                        break;
                                    default:
                                        break;
                                }
                            }
                            #endregion


                        }
                    }
                }
            }

            //if (string.IsNullOrEmpty(feedItem.Title))
            //    feedItem.Title = GetTitleFromHtmlDocument(doc);

            //feedItem.Title = feedItem.Title.Trim();



            return feedItem;




        }

        private static string GetContentFromMetaTag(HtmlNode metaNode)
        {
            try
            {
                if (metaNode != null && metaNode.Attributes != null && metaNode.Attributes.Count>0 && metaNode.Attributes["content"]!=null && metaNode.Attributes["content"].Value != null)
                    return metaNode.Attributes["content"].Value;
                else
                    return null;
            }
            catch (Exception ex)
            {
                TempLogger.Log("Error in GetContentFromMetaTag ", ex);
                return null;
            }
        }

        public static string GetTitleFromHtmlDocument(HtmlDocument doc)
        {
            try
            {
                string titleXpath = "/html/head/title";

                return ExtractStringValueFromHtmlNodeByXpath(doc, titleXpath);
            }
            catch (Exception ex)
            {
                TempLogger.Log("Error in GetTitleFromHtmlDocument ", ex);
                return null;
            }
        }

        private static string ExtractStringValueFromHtmlNodeByXpath(HtmlDocument doc, string xPath)
        {
            try
            {
                if (doc == null || doc.DocumentNode == null)
                    return null;

                var node = doc.DocumentNode.SelectSingleNode(xPath);

                if (node != null)
                {
                    return node.InnerText;
                }
                else
                    return null;
            }
            catch (Exception ex)
            {
                TempLogger.Log("Error in ExtractStringValueFromHtmlNodeByXpath ", ex);
                return null;
            }
        }

        public  static void ReadFeed()
        {
            string url = "http://www.facebook.com/feeds/page.php?format=atom10&id=147782351928125";
            url = "http://gdata.youtube.com/feeds/base/users/tv9maharashtra/uploads?alt=rss&v=2&orderby=published";

            var req = (HttpWebRequest)WebRequest.Create(url);
            req.Method = "GET";
            req.UserAgent = "Fiddler";


            var rep = req.GetResponse();
            var reader = XmlReader.Create(rep.GetResponseStream());

            SyndicationFeed feed = SyndicationFeed.Load(reader);


            reader.Close();
            rep.Close();

            foreach (SyndicationItem item in feed.Items)
                Console.WriteLine("#### {0}", item.Title.Text);
        }

    }
}
